from config import *


class Runner:
    start_at = time.time()

    def __init__(self, timeout=None):
        self.debug = debug
        self.timeout = timeout

    def run(self):
        for source in spider_dict:
            spider = spider_dict[source]['spider'](img_dir, debug)
            if self.debug:
                spider.set_origin(source, "debug")
                spider.start(debug_file_path + source + "/list.txt")
            else:
                for column, url in spider_dict[source]['urls'].items():
                    self.check_timeout()
                    spider.set_origin(source, column)
                    spider.start(url)

    def check_timeout(self):
        if self.timeout is not None and (time.time() - self.start_at) > self.timeout:
            raise Exception("TimeOut!")


if __name__ == '__main__':
    runner = Runner()
    runner.run()
